{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import json\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = OrderedDict({    \n",
    "    '0_model=BaseModel,network=resnet152,percent_on_k_winner=0.25': 'Resnet-Dense-Kwinners',\n",
    "    '1_model=SparseModel,network=resnet152,percent_on_k_winner=0.25': 'Resnet-Sparse-Kwinners',\n",
    "    '2_model=BaseModel,network=WideResNet,percent_on_k_winner=0.25': 'WideResnet-Dense-Kwinners',\n",
    "    '3_model=SparseModel,network=WideResNet,percent_on_k_winner=0.25': 'WideResnet-Sparse-Kwinners',\n",
    "    '4_model=BaseModel,network=resnet152,percent_on_k_winner=1': 'Resnet-Dense-ReLU',\n",
    "    '5_model=SparseModel,network=resnet152,percent_on_k_winner=1': 'Resnet-Sparse-ReLU',\n",
    "    '6_model=BaseModel,network=WideResNet,percent_on_k_winner=1': 'WideResnet-Dense-ReLU',\n",
    "    '7_model=SparseModel,network=WideResNet,percent_on_k_winner=1': 'WideResnet-Sparse-ReLU',\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['WideResnet-Dense-Kwinners', 'Resnet-Dense-ReLU',\n",
       "       'WideResnet-Sparse-Kwinners', 'Resnet-Sparse-Kwinners',\n",
       "       'WideResnet-Sparse-ReLU', 'Resnet-Dense-Kwinners',\n",
       "       'WideResnet-Dense-ReLU', 'Resnet-Sparse-ReLU'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.expanduser(\"~/nta/results/resnet_cifar2/noise_results.json\")\n",
    "with open(file, 'r') as f:\n",
    "    noise_results = json.load(f)\n",
    "\n",
    "mapped_results = {}\n",
    "for k,v in noise_results.items():\n",
    "    new_k = mapping[k]\n",
    "    mapped_results[new_k] = v\n",
    "  \n",
    "df = pd.DataFrame.from_dict(mapped_results)\n",
    "df = df.drop('0.2')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.41</td>\n",
       "      <td>91.92</td>\n",
       "      <td>94.76</td>\n",
       "      <td>94.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>87.20</td>\n",
       "      <td>86.60</td>\n",
       "      <td>85.41</td>\n",
       "      <td>85.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>78.52</td>\n",
       "      <td>77.98</td>\n",
       "      <td>74.44</td>\n",
       "      <td>74.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>66.61</td>\n",
       "      <td>67.99</td>\n",
       "      <td>61.18</td>\n",
       "      <td>62.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>54.88</td>\n",
       "      <td>57.37</td>\n",
       "      <td>49.03</td>\n",
       "      <td>51.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>44.84</td>\n",
       "      <td>47.29</td>\n",
       "      <td>39.22</td>\n",
       "      <td>42.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>36.66</td>\n",
       "      <td>38.73</td>\n",
       "      <td>31.87</td>\n",
       "      <td>35.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>30.71</td>\n",
       "      <td>31.67</td>\n",
       "      <td>26.23</td>\n",
       "      <td>30.28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Dense-ReLU  Resnet-Sparse-Kwinners  WideResnet-Dense-ReLU  \\\n",
       "0                  93.41                   91.92                  94.76   \n",
       "0.025              87.20                   86.60                  85.41   \n",
       "0.05               78.52                   77.98                  74.44   \n",
       "0.075              66.61                   67.99                  61.18   \n",
       "0.1                54.88                   57.37                  49.03   \n",
       "0.125              44.84                   47.29                  39.22   \n",
       "0.15               36.66                   38.73                  31.87   \n",
       "0.175              30.71                   31.67                  26.23   \n",
       "\n",
       "       WideResnet-Sparse-Kwinners  \n",
       "0                           94.14  \n",
       "0.025                       85.96  \n",
       "0.05                        74.76  \n",
       "0.075                       62.44  \n",
       "0.1                         51.22  \n",
       "0.125                       42.31  \n",
       "0.15                        35.24  \n",
       "0.175                       30.28  "
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# paper comparison table\n",
    "(df[['Resnet-Dense-ReLU', 'Resnet-Sparse-Kwinners', 'WideResnet-Dense-ReLU', 'WideResnet-Sparse-Kwinners']]*100).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Dense-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Dense-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.41</td>\n",
       "      <td>93.39</td>\n",
       "      <td>94.76</td>\n",
       "      <td>94.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>87.20</td>\n",
       "      <td>88.16</td>\n",
       "      <td>85.41</td>\n",
       "      <td>85.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>78.52</td>\n",
       "      <td>79.92</td>\n",
       "      <td>74.44</td>\n",
       "      <td>75.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>66.61</td>\n",
       "      <td>69.09</td>\n",
       "      <td>61.18</td>\n",
       "      <td>62.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>54.88</td>\n",
       "      <td>57.89</td>\n",
       "      <td>49.03</td>\n",
       "      <td>49.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>44.84</td>\n",
       "      <td>48.14</td>\n",
       "      <td>39.22</td>\n",
       "      <td>38.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>36.66</td>\n",
       "      <td>39.90</td>\n",
       "      <td>31.87</td>\n",
       "      <td>30.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>30.71</td>\n",
       "      <td>33.72</td>\n",
       "      <td>26.23</td>\n",
       "      <td>25.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Dense-ReLU  Resnet-Dense-Kwinners  WideResnet-Dense-ReLU  \\\n",
       "0                  93.41                  93.39                  94.76   \n",
       "0.025              87.20                  88.16                  85.41   \n",
       "0.05               78.52                  79.92                  74.44   \n",
       "0.075              66.61                  69.09                  61.18   \n",
       "0.1                54.88                  57.89                  49.03   \n",
       "0.125              44.84                  48.14                  39.22   \n",
       "0.15               36.66                  39.90                  31.87   \n",
       "0.175              30.71                  33.72                  26.23   \n",
       "\n",
       "       WideResnet-Dense-Kwinners  \n",
       "0                          94.62  \n",
       "0.025                      85.68  \n",
       "0.05                       75.11  \n",
       "0.075                      62.61  \n",
       "0.1                        49.31  \n",
       "0.125                      38.17  \n",
       "0.15                       30.23  \n",
       "0.175                      25.00  "
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dense weights only\n",
    "(df[['Resnet-Dense-ReLU', 'Resnet-Dense-Kwinners', 'WideResnet-Dense-ReLU', 'WideResnet-Dense-Kwinners']]*100).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Sparse-ReLU</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Sparse-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>92.97</td>\n",
       "      <td>91.92</td>\n",
       "      <td>94.22</td>\n",
       "      <td>94.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>88.02</td>\n",
       "      <td>86.60</td>\n",
       "      <td>85.10</td>\n",
       "      <td>85.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>80.20</td>\n",
       "      <td>77.98</td>\n",
       "      <td>74.05</td>\n",
       "      <td>74.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>71.08</td>\n",
       "      <td>67.99</td>\n",
       "      <td>62.11</td>\n",
       "      <td>62.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>62.10</td>\n",
       "      <td>57.37</td>\n",
       "      <td>51.57</td>\n",
       "      <td>51.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>54.31</td>\n",
       "      <td>47.29</td>\n",
       "      <td>42.04</td>\n",
       "      <td>42.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>47.39</td>\n",
       "      <td>38.73</td>\n",
       "      <td>35.43</td>\n",
       "      <td>35.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>42.03</td>\n",
       "      <td>31.67</td>\n",
       "      <td>30.17</td>\n",
       "      <td>30.28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Sparse-ReLU  Resnet-Sparse-Kwinners  WideResnet-Sparse-ReLU  \\\n",
       "0                   92.97                   91.92                   94.22   \n",
       "0.025               88.02                   86.60                   85.10   \n",
       "0.05                80.20                   77.98                   74.05   \n",
       "0.075               71.08                   67.99                   62.11   \n",
       "0.1                 62.10                   57.37                   51.57   \n",
       "0.125               54.31                   47.29                   42.04   \n",
       "0.15                47.39                   38.73                   35.43   \n",
       "0.175               42.03                   31.67                   30.17   \n",
       "\n",
       "       WideResnet-Sparse-Kwinners  \n",
       "0                           94.14  \n",
       "0.025                       85.96  \n",
       "0.05                        74.76  \n",
       "0.075                       62.44  \n",
       "0.1                         51.22  \n",
       "0.125                       42.31  \n",
       "0.15                        35.24  \n",
       "0.175                       30.28  "
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sparse weights only\n",
    "(df[['Resnet-Sparse-ReLU', 'Resnet-Sparse-Kwinners', 'WideResnet-Sparse-ReLU', 'WideResnet-Sparse-Kwinners']]*100).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Sparse-ReLU</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-ReLU</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.41</td>\n",
       "      <td>92.97</td>\n",
       "      <td>94.76</td>\n",
       "      <td>94.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>87.20</td>\n",
       "      <td>88.02</td>\n",
       "      <td>85.41</td>\n",
       "      <td>85.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>78.52</td>\n",
       "      <td>80.20</td>\n",
       "      <td>74.44</td>\n",
       "      <td>74.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>66.61</td>\n",
       "      <td>71.08</td>\n",
       "      <td>61.18</td>\n",
       "      <td>62.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>54.88</td>\n",
       "      <td>62.10</td>\n",
       "      <td>49.03</td>\n",
       "      <td>51.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>44.84</td>\n",
       "      <td>54.31</td>\n",
       "      <td>39.22</td>\n",
       "      <td>42.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>36.66</td>\n",
       "      <td>47.39</td>\n",
       "      <td>31.87</td>\n",
       "      <td>35.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>30.71</td>\n",
       "      <td>42.03</td>\n",
       "      <td>26.23</td>\n",
       "      <td>30.17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Dense-ReLU  Resnet-Sparse-ReLU  WideResnet-Dense-ReLU  \\\n",
       "0                  93.41               92.97                  94.76   \n",
       "0.025              87.20               88.02                  85.41   \n",
       "0.05               78.52               80.20                  74.44   \n",
       "0.075              66.61               71.08                  61.18   \n",
       "0.1                54.88               62.10                  49.03   \n",
       "0.125              44.84               54.31                  39.22   \n",
       "0.15               36.66               47.39                  31.87   \n",
       "0.175              30.71               42.03                  26.23   \n",
       "\n",
       "       WideResnet-Sparse-ReLU  \n",
       "0                       94.22  \n",
       "0.025                   85.10  \n",
       "0.05                    74.05  \n",
       "0.075                   62.11  \n",
       "0.1                     51.57  \n",
       "0.125                   42.04  \n",
       "0.15                    35.43  \n",
       "0.175                   30.17  "
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# relu only\n",
    "(df[['Resnet-Dense-ReLU', 'Resnet-Sparse-ReLU', 'WideResnet-Dense-ReLU', 'WideResnet-Sparse-ReLU']]*100).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-Kwinners</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Dense-Kwinners</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93.39</td>\n",
       "      <td>91.92</td>\n",
       "      <td>94.62</td>\n",
       "      <td>94.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>88.16</td>\n",
       "      <td>86.60</td>\n",
       "      <td>85.68</td>\n",
       "      <td>85.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>79.92</td>\n",
       "      <td>77.98</td>\n",
       "      <td>75.11</td>\n",
       "      <td>74.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>69.09</td>\n",
       "      <td>67.99</td>\n",
       "      <td>62.61</td>\n",
       "      <td>62.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>57.89</td>\n",
       "      <td>57.37</td>\n",
       "      <td>49.31</td>\n",
       "      <td>51.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>48.14</td>\n",
       "      <td>47.29</td>\n",
       "      <td>38.17</td>\n",
       "      <td>42.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>39.90</td>\n",
       "      <td>38.73</td>\n",
       "      <td>30.23</td>\n",
       "      <td>35.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>33.72</td>\n",
       "      <td>31.67</td>\n",
       "      <td>25.00</td>\n",
       "      <td>30.28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Dense-Kwinners  Resnet-Sparse-Kwinners  \\\n",
       "0                      93.39                   91.92   \n",
       "0.025                  88.16                   86.60   \n",
       "0.05                   79.92                   77.98   \n",
       "0.075                  69.09                   67.99   \n",
       "0.1                    57.89                   57.37   \n",
       "0.125                  48.14                   47.29   \n",
       "0.15                   39.90                   38.73   \n",
       "0.175                  33.72                   31.67   \n",
       "\n",
       "       WideResnet-Dense-Kwinners  WideResnet-Sparse-Kwinners  \n",
       "0                          94.62                       94.14  \n",
       "0.025                      85.68                       85.96  \n",
       "0.05                       75.11                       74.76  \n",
       "0.075                      62.61                       62.44  \n",
       "0.1                        49.31                       51.22  \n",
       "0.125                      38.17                       42.31  \n",
       "0.15                       30.23                       35.24  \n",
       "0.175                      25.00                       30.28  "
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# kwinners only\n",
    "(df[['Resnet-Dense-Kwinners', 'Resnet-Sparse-Kwinners', 'WideResnet-Dense-Kwinners', 'WideResnet-Sparse-Kwinners']]*100).round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping = OrderedDict({    \n",
    "    'Trainable_0_model=BaseModel,network=resnet152,percent_on_k_winner=0.25_2019-10-18_22-51-438_axbl42': 'Resnet-Dense-Kwinners',\n",
    "    'Trainable_1_model=SparseModel,network=resnet152,percent_on_k_winner=0.25_2019-10-18_22-51-450nnmjnv4': 'Resnet-Sparse-Kwinners',\n",
    "    'Trainable_2_model=BaseModel,network=WideResNet,percent_on_k_winner=0.25_2019-10-18_22-51-45w75kg1i1': 'WideResnet-Dense-Kwinners',\n",
    "    'Trainable_3_model=SparseModel,network=WideResNet,percent_on_k_winner=0.25_2019-10-18_22-51-45ncthm6wh': 'WideResnet-Sparse-Kwinners',\n",
    "    'Trainable_4_model=BaseModel,network=resnet152,percent_on_k_winner=1_2019-10-18_22-51-45tbdn5tcn': 'Resnet-Dense-ReLU',\n",
    "    'Trainable_5_model=SparseModel,network=resnet152,percent_on_k_winner=1_2019-10-18_22-51-45l5vvkizn': 'Resnet-Sparse-ReLU',\n",
    "    'Trainable_6_model=BaseModel,network=WideResNet,percent_on_k_winner=1_2019-10-18_22-51-45gcbcbn3z': 'WideResnet-Dense-ReLU',\n",
    "    'Trainable_7_model=SparseModel,network=WideResNet,percent_on_k_winner=1_2019-10-18_22-51-451rb5spc0': 'WideResnet-Sparse-ReLU',\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Resnet-Dense-ReLU', 'WideResnet-Sparse-ReLU', 'WideResnet-Dense-ReLU',\n",
       "       'Resnet-Sparse-Kwinners', 'WideResnet-Dense-Kwinners',\n",
       "       'Resnet-Sparse-ReLU', 'Resnet-Dense-Kwinners',\n",
       "       'WideResnet-Sparse-Kwinners'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = os.path.expanduser(\"~/nta/results/resnet_cifar1/noise_results.json\")\n",
    "with open(file, 'r') as f:\n",
    "  noise_results = json.load(f)\n",
    "\n",
    "mapped_results = {}\n",
    "for k,v in noise_results.items():\n",
    "  new_k = mapping[k]\n",
    "  mapped_results[new_k] = v\n",
    "  \n",
    "df = pd.DataFrame.from_dict(mapped_results)\n",
    "df = df.drop('0.2')\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Resnet-Dense-ReLU</th>\n",
       "      <th>Resnet-Sparse-Kwinners</th>\n",
       "      <th>WideResnet-Dense-ReLU</th>\n",
       "      <th>WideResnet-Sparse-Kwinners</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>94.54</td>\n",
       "      <td>92.17</td>\n",
       "      <td>95.02</td>\n",
       "      <td>93.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.025</th>\n",
       "      <td>88.68</td>\n",
       "      <td>85.44</td>\n",
       "      <td>85.35</td>\n",
       "      <td>85.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>79.13</td>\n",
       "      <td>75.22</td>\n",
       "      <td>74.53</td>\n",
       "      <td>73.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.075</th>\n",
       "      <td>66.38</td>\n",
       "      <td>63.27</td>\n",
       "      <td>61.85</td>\n",
       "      <td>61.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.1</th>\n",
       "      <td>53.81</td>\n",
       "      <td>52.42</td>\n",
       "      <td>48.97</td>\n",
       "      <td>47.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.125</th>\n",
       "      <td>43.23</td>\n",
       "      <td>43.42</td>\n",
       "      <td>39.11</td>\n",
       "      <td>36.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.15</th>\n",
       "      <td>35.42</td>\n",
       "      <td>36.47</td>\n",
       "      <td>31.02</td>\n",
       "      <td>27.87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.175</th>\n",
       "      <td>30.39</td>\n",
       "      <td>31.47</td>\n",
       "      <td>26.26</td>\n",
       "      <td>22.65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Resnet-Dense-ReLU  Resnet-Sparse-Kwinners  WideResnet-Dense-ReLU  \\\n",
       "0                  94.54                   92.17                  95.02   \n",
       "0.025              88.68                   85.44                  85.35   \n",
       "0.05               79.13                   75.22                  74.53   \n",
       "0.075              66.38                   63.27                  61.85   \n",
       "0.1                53.81                   52.42                  48.97   \n",
       "0.125              43.23                   43.42                  39.11   \n",
       "0.15               35.42                   36.47                  31.02   \n",
       "0.175              30.39                   31.47                  26.26   \n",
       "\n",
       "       WideResnet-Sparse-Kwinners  \n",
       "0                           93.75  \n",
       "0.025                       85.48  \n",
       "0.05                        73.96  \n",
       "0.075                       61.33  \n",
       "0.1                         47.61  \n",
       "0.125                       36.12  \n",
       "0.15                        27.87  \n",
       "0.175                       22.65  "
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# paper comparison table\n",
    "df[['Resnet-Dense-ReLU', 'Resnet-Sparse-Kwinners', 'WideResnet-Dense-ReLU', 'WideResnet-Sparse-Kwinners']]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
